{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "from gensim import corpora, models, similarities\n",
    "import jieba.posseg as pseg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Dumping model to file cache C:\\Users\\ldq\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.773 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    }
   ],
   "source": [
    "corpus=[\"我非常喜欢看电视剧\",\"我非常喜欢旅行\",\"我非常喜欢吃苹果\", '我非常喜欢跑步', '王者荣耀春季赛开战啦']\n",
    "\n",
    "jieba.add_word('王者荣耀', tag='n')\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# TF-IDF生成步骤\n",
    "# Step1, 生成字典\n",
    "dictionary = corpora.Dictionary(train)\n",
    "# Step2, 生成语料 \n",
    "corpus = [dictionary.doc2bow(text) for text in train]\n",
    "# Step3, 定义TFIDF模型 \n",
    "tfidf_model = models.TfidfModel(corpus, dictionary=dictionary)\n",
    "# Step4, 生成TFIDF矩阵 \n",
    "corpus_tfidf = tfidf_model[corpus]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['我', '非常', '喜欢', '看', '电视剧'],\n",
       " ['我', '非常', '喜欢', '旅行'],\n",
       " ['我', '非常', '喜欢', '吃', '苹果'],\n",
       " ['我', '非常', '喜欢', '跑步'],\n",
       " ['王者荣耀', '春季', '赛', '开战', '啦']]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 分词\n",
    "text = [[word for word in jieba.cut(words)] for words in corpus]\n",
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<gensim.corpora.dictionary.Dictionary at 0x29aef100790>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#生成字典\n",
    "dictionary = corpora.Dictionary(text)\n",
    "print(dictionary)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary(14 unique tokens: ['喜欢', '我', '电视剧', '看', '非常']...)\n"
     ]
    }
   ],
   "source": [
    "print(dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 4, 4: 4, 0: 4, 3: 1, 2: 1, 5: 1, 6: 1, 7: 1, 8: 1, 12: 1, 11: 1, 13: 1, 10: 1, 9: 1}\n"
     ]
    }
   ],
   "source": [
    "print(dictionary.dfs)# 字典词频，{单词id，在多少文档中出现}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dictionary.num_pos  # 所有词的个数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "featurenum = len(dictionary.token2id.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'喜欢': 0,\n",
       " '我': 1,\n",
       " '电视剧': 2,\n",
       " '看': 3,\n",
       " '非常': 4,\n",
       " '旅行': 5,\n",
       " '吃': 6,\n",
       " '苹果': 7,\n",
       " '跑步': 8,\n",
       " '啦': 9,\n",
       " '开战': 10,\n",
       " '春季': 11,\n",
       " '王者荣耀': 12,\n",
       " '赛': 13}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dictionary.token2id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[(0, 1), (1, 1), (2, 1), (3, 1), (4, 1)],\n",
       " [(0, 1), (1, 1), (4, 1), (5, 1)],\n",
       " [(0, 1), (1, 1), (4, 1), (6, 1), (7, 1)],\n",
       " [(0, 1), (1, 1), (4, 1), (8, 1)],\n",
       " [(9, 1), (10, 1), (11, 1), (12, 1), (13, 1)]]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 生成语料\n",
    "corpus = [dictionary.doc2bow(txt) for txt in text]\n",
    "corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tfidf_model= TfidfModel(num_docs=5, num_nnz=23)\n"
     ]
    }
   ],
   "source": [
    "# 训练tfidf\n",
    "tfidf_model = models.TfidfModel(corpus, dictionary=dictionary)\n",
    "print('tfidf_model=', tfidf_model) # 只要记录BOW矩阵的非零元素个数(num_nnz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "corpus= [[(0, 1), (1, 1), (2, 1), (3, 1), (4, 1)], [(0, 1), (1, 1), (4, 1), (5, 1)], [(0, 1), (1, 1), (4, 1), (6, 1), (7, 1)], [(0, 1), (1, 1), (4, 1), (8, 1)], [(9, 1), (10, 1), (11, 1), (12, 1), (13, 1)]]\n",
      "转换整个语料库：\n",
      "[(0, 0.09665456461747607), (1, 0.09665456461747607), (2, 0.6971275655917712), (3, 0.6971275655917712), (4, 0.09665456461747607)]\n",
      "[(0, 0.1348140795693492), (1, 0.1348140795693492), (4, 0.1348140795693492), (5, 0.9723556406220963)]\n",
      "[(0, 0.09665456461747605), (1, 0.09665456461747605), (4, 0.09665456461747605), (6, 0.6971275655917711), (7, 0.6971275655917711)]\n",
      "[(0, 0.1348140795693492), (1, 0.1348140795693492), (4, 0.1348140795693492), (8, 0.9723556406220963)]\n",
      "[(9, 0.447213595499958), (10, 0.447213595499958), (11, 0.447213595499958), (12, 0.447213595499958), (13, 0.447213595499958)]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# 得到每个单词的TF-IDF值\n",
    "corpus_tfidf = tfidf_model[corpus]\n",
    "print('corpus=', corpus)\n",
    "print('转换整个语料库：')\n",
    "for doc in corpus_tfidf:\n",
    "    print(doc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<gensim.similarities.docsim.SparseMatrixSimilarity at 0x29aef038c40>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 生成余弦相似度索引, 使用SparseMatrixSimilarity()，可以占用更少的内存和磁盘空间。\n",
    "index = similarities.SparseMatrixSimilarity(corpus_tfidf, num_features=featurenum) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<gensim.similarities.docsim.SparseMatrixSimilarity object at 0x0000029AEF038C40>\n"
     ]
    }
   ],
   "source": [
    "print(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['我', '喜欢', '看', '电视剧']\n"
     ]
    }
   ],
   "source": [
    "# 测试阶段   模型对测试集进行operation；求余弦相似度。对于给定的新文本，找到训练集中最相似的五篇文章作为推荐\n",
    "\n",
    "test = jieba.lcut('我喜欢看电视剧')\n",
    "print(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 1), (1, 1), (2, 1), (3, 1)]\n"
     ]
    }
   ],
   "source": [
    "# 生成bow向量\n",
    "ver = dictionary.doc2bow(test)\n",
    "print(ver)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0.09710923130609715), (1, 0.09710923130609715), (2, 0.7004068797457225), (3, 0.7004068797457225)]\n"
     ]
    }
   ],
   "source": [
    "# 生成tfidf向量\n",
    "test_vec = tfidf_model[ver]\n",
    "print(test_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.99531806 0.02618338 0.0187721  0.02618338 0.        ]\n"
     ]
    }
   ],
   "source": [
    "# 返回test_vec 和训练语料中所有文本的余弦相似度。返回结果是个numpy数组\n",
    "print(index.get_similarities(test_vec))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, '0.164*\"非常\" + 0.149*\"喜欢\" + 0.135*\"我\" + 0.077*\"电视剧\" + 0.076*\"看\"')\n",
      "(1, '0.136*\"我\" + 0.122*\"喜欢\" + 0.107*\"非常\" + 0.074*\"跑步\" + 0.072*\"啦\"')\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\"\"\" 继续 LDA模型 \"\"\"\n",
    "lda = models.ldamodel.LdaModel(corpus = corpus, id2word=dictionary, num_topics=2)\n",
    "for topic in lda.print_topics(num_words=5):\n",
    "\tprint(topic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([[5.359866  , 0.6401258 ],\n",
      "       [4.260902  , 0.73909175],\n",
      "       [5.204812  , 0.7951776 ],\n",
      "       [1.4650187 , 3.534974  ],\n",
      "       [0.5558902 , 5.444096  ]], dtype=float32), None)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# 主题推断\n",
    "print(lda.inference(corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bow: [(0, 1), (1, 1), (3, 1), (12, 1)]\n"
     ]
    }
   ],
   "source": [
    "text5 = '我喜欢看王者荣耀KPL挑战赛'\n",
    "# bow 向量\n",
    "bow = dictionary.doc2bow([word for word in jieba.cut(text5)])\n",
    "print('bow:', bow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "我喜欢看王者荣耀KPL挑战赛\n",
      "主题0 推断值2.9935219287872314\n",
      "\n",
      "主题1 推断值2.00646710395813\n",
      "\n"
     ]
    }
   ],
   "source": [
    "inference_result = lda.inference([bow])[0]\n",
    "print(text5)\n",
    "for e, value in enumerate(inference_result[0]):\n",
    "\tprint('主题{} 推断值{}\\n'.format(e, value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\n"
     ]
    }
   ],
   "source": [
    "# 得到向量ID\n",
    "word = '王者荣耀'\n",
    "word_id = dictionary.doc2idx([word])[0]\n",
    "print(word_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0.014529371), (1, 0.04830328)]\n",
      "王者荣耀与主题0的关系值为1.4529370702803135%\n",
      "王者荣耀与主题1的关系值为4.83032800257206%\n"
     ]
    }
   ],
   "source": [
    "# 得到指定单词与主题的关系\n",
    "print(lda.get_term_topics(word_id))\n",
    "for i in lda.get_term_topics(word_id):\n",
    "\tprint('{}与主题{}的关系值为{}%'.format(word, i[0], i[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(4, 0.16418037), (0, 0.1492235), (1, 0.1345312), (2, 0.07704167), (3, 0.076336406), (5, 0.06528607), (7, 0.06034895), (6, 0.058102503), (13, 0.03813204), (11, 0.036385816)]\n"
     ]
    }
   ],
   "source": [
    "# 查看主题0的重要词汇\n",
    "print(lda.get_topic_terms(0, topn=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
